package engine

import (
	"log"
	"spider/models"
	"time"
)

func Run(itemsChan chan interface{},rateLimiter <-chan time.Time,seeds ...models.Request)  {
	var requests []models.Request
	for _,r:=range seeds{
		requests=append(requests,r)
	}

	var errOccurNum  int
	var errOccurMax int
	//最大错误次数
	errOccurMax=2000

	for len(requests)>0{
		<-rateLimiter
		r:=requests[0]
		requests=requests[1:]

		body,err:=r.ReqFunc(r.Url,r.Header,r.PostData)
		log.Printf("Fetch url=%s\n ",r.Url)
		if err!=nil{
			if len(requests)<1 {
				break
			}
			requests=requests[1:]
			errOccurNum++
			if errOccurNum>errOccurMax{
				break
			}
			continue
		}
		parseResult,err:=r.Parser(body,r.DataFromParent)
		if err!=nil{
			if len(requests)<1 {
				break
			}
			requests=requests[1:]

			//log.Printf("parse err=%s, body=%s\n",err,body)
			errOccurNum++
			log.Println("parse err ",errOccurNum,r.Url)
			if errOccurNum>errOccurMax{
				break
			}
			continue
		}

		toDoRequestLen:=len(parseResult.Requests)
		if toDoRequestLen >0{
			requests= append(requests, parseResult.Requests...)
		}
		if r.ParserName=="ParseMeituanList"{
			log.Printf("[url=%s]\n",r.Url)
		}

		if r.ParserName =="ParsePopOne"{
			  PushItems(parseResult.Items,itemsChan)
		}
	}
}
func PushItems(items interface{},itemsChan chan interface{})  {
	itemsChan<-items
}
